{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from PIL import Image\n",
    "from PIL import ImageDraw\n",
    "import tensorflow as tf\n",
    "import random\n",
    "from collections import deque\n",
    "plt.style.use({'figure.figsize':(10, 10)})\n",
    "pd.set_option('max_rows', 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "States=np.array(['L60D0/30/H0','L180D0/30/H0','L250D0/30/H0','L300D0/30/H0',\n",
    "'L60D30/60/H0','L180D30/60/H0','L250D30/60/H0','L300D30/60/H0',\n",
    "'L60D60/90/H0','L180D60/90/H0','L250D60/90/H0','L300D60/90/H0',\n",
    "'L60D90/120/H0','L180D90/120/H0','L250D90/120/H0','L300D90/120/H0',\n",
    "'L60D120/150/H0','L180D120/150/H0','L250D120/150/H0','L300D120/150/H0',\n",
    "'L60D150/180/H0','L180D150/180/H0','L250D150/180/H0','L300D150/180/H0',\n",
    "'L60D180/210/H0','L180D180/210/H0','L250D180/210/H0','L300D180/210/H0',\n",
    "'L60D210/240/H0','L180D210/240/H0','L250D210/240/H0','L300D210/240/H0',\n",
    "'L60D240/270/H0','L180D240/270/H0','L250D240/270/H0','L300D240/270/H0',\n",
    "'L60D270/300/H0','L180D270/300/H0','L250D270/300/H0','L300D270/300/H0',\n",
    "'L60D300/330/H0','L180D300/330/H0','L250D300/330/H0','L300D300/330/H0',\n",
    "'L60D330/360/H0','L180D330/360/H0','L250D330/360/H0','L300D330/360/H0',\n",
    "'L60D0/30/H1','L180D0/30/H1','L250D0/30/H1','L300D0/30/H1',\n",
    "'L60D30/60/H1','L180D30/60/H1','L250D30/60/H1','L300D30/60/H1',\n",
    "'L60D60/90/H1','L180D60/90/H1','L250D60/90/H1','L300D60/90/H1',\n",
    "'L60D90/120/H1','L180D90/120/H1','L250D90/120/H1','L300D90/120/H1',\n",
    "'L60D120/150/H1','L180D120/150/H1','L250D120/150/H1','L300D120/150/H1',\n",
    "'L60D150/180/H1','L180D150/180/H1','L250D150/180/H1','L300D150/180/H1',\n",
    "'L60D180/210/H1','L180D180/210/H1','L250D180/210/H1','L300D180/210/H1',\n",
    "'L60D210/240/H1','L180D210/240/H1','L250D210/240/H1','L300D210/240/H1',\n",
    "'L60D240/270/H1','L180D240/270/H1','L250D240/270/H1','L300D240/270/H1',\n",
    "'L60D270/300/H1','L180D270/300/H1','L250D270/300/H1','L300D270/300/H1',\n",
    "'L60D300/330/H1','L180D300/330/H1','L250D300/330/H1','L300D300/330/H1',\n",
    "'L60D330/360/H1','L180D330/360/H1','L250D330/360/H1','L300D330/360/H1',\n",
    "'L60D0/30/H2','L180D0/30/H2','L250D0/30/H2','L300D0/30/H2',\n",
    "'L60D30/60/H2','L180D30/60/H2','L250D30/60/H2','L300D30/60/H2',\n",
    "'L60D60/90/H2','L180D60/90/H2','L250D60/90/H2','L300D60/90/H2',\n",
    "'L60D90/120/H2','L180D90/120/H2','L250D90/120/H2','L300D90/120/H2',\n",
    "'L60D120/150/H2','L180D120/150/H2','L250D120/150/H2','L300D120/150/H2',\n",
    "'L60D150/180/H2','L180D150/180/H2','L250D150/180/H2','L300D150/180/H2',\n",
    "'L60D180/210/H2','L180D180/210/H2','L250D180/210/H2','L300D180/210/H2',\n",
    "'L60D210/240/H2','L180D210/240/H2','L250D210/240/H2','L300D210/240/H2',\n",
    "'L60D240/270/H2','L180D240/270/H2','L250D240/270/H2','L300D240/270/H2',\n",
    "'L60D270/300/H2','L180D270/300/H2','L250D270/300/H2','L300D270/300/H2',\n",
    "'L60D300/330/H2','L180D300/330/H2','L250D300/330/H2','L300D300/330/H2',\n",
    "'L60D330/360/H2','L180D330/360/H2','L250D330/360/H2','L300D330/360/H2'])\n",
    "Actions=np.array(['FrontH0','FrontH1','FrontH2','Left_45D','Right_45D'])\n",
    "Length_state=len(States)\n",
    "Length_action=len(Actions)\n",
    "Action_list=np.identity(Length_action)\n",
    "State_list=np.identity(Length_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Epsilon_start=1\n",
    "Epsilon_final=0.01\n",
    "Decay_Rate=0.000001\n",
    "Velocity_tripod=0.289*40\n",
    "Velocity_quadruped=0.115*40\n",
    "Velocity_onebyone=0.096*40\n",
    "Up_degree=np.array([-40,-20,0,20,40])\n",
    "Left_degree=np.array([-60,-80,-100,-120])\n",
    "Right_degree=np.array([60,80,100,120]) #\n",
    "Robot_r=40 #the radius of hexapod robot\n",
    "Beta=0.9\n",
    "learning_rate=0.01\n",
    "Observe=1000\n",
    "Store_memory=deque()\n",
    "Memory_size=5000\n",
    "Batch_size=20\n",
    "Epoche=0\n",
    "Action_times=0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the map, 'RGB' mode is used to reprensent different colors.\n",
    "\n",
    "(0,0,0)---Black---Obstacle and margin\n",
    "\n",
    "(255,255,255)---White---Flat ground\n",
    "\n",
    "(255,0,0)---Red---Destination\n",
    "\n",
    "(0,255,255)---Cyan---Hexapod robot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlsAAAJCCAYAAAD3HAIiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGZdJREFUeJzt3W+obXed3/HP18SxYVQamzZkbgJmaBqI0omTPw2MDLbD\njBmfRJ/I9cEYqJgBU1GYZriZgUZbpOlYHZDW0CuKsTiGgErCMLbEIEihMYk2mn9zx6tRzCUmpFJi\nn2RM/PbBWbF7rvfPueee79nnnrxesDnr/PZe+/z2yubmzVp77VXdHQAAZrxi3RMAANjLxBYAwCCx\nBQAwSGwBAAwSWwAAg8QWAMCgHY+tqrq2qg5V1eGqOrDTfx8AYCfVTn7PVlWdleRvkvxukieTPJDk\nXd392I5NAgBgB+30nq2rkxzu7u93998muSPJdTs8BwCAHXP2Dv+9fUl+tPL7k0n+2dEPqqobktyw\n/HrFDswLAOAXuru267l2OrY2pbsPJjmYJFXlekIAwBlrpw8jHkly0crvFy5jAAB70k7H1gNJLqmq\ni6vqV5LsT3L3Ds8BAGDH7OhhxO5+oar+VZL/nuSsJJ/p7kd3cg4AADtpR7/6YSt8ZgsA2Gnb+QF5\n3yAPADBoV56NuFm7fa8cALB7VW3bzqsTsmcLAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAA\nBoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAA\nBoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFnr3sCAMDmfPijt657\nCr/klpsOrHsKu549WwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADPKlpvzCA88e\nWvcURlx13qXrngIAL2P2bAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMMiXmsIZ\n6sMfvXXdU4BfuOWmA+ueAuxa9mwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBI\nbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBI\nbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADDo\n7HVPAIANt/zxzb9Y/vCf/fs1zgTYTqe1Z6uqflBVD1fVQ1X14DL2uqq6p6q+u/w8d+XxN1fV4ao6\nVFVvPd3JA+wFt/zxzX8ntF4aA/aG7TiM+M+7+/LuvnL5/UCSe7v7kiT3Lr+nqi5Lsj/JG5Jcm+ST\nVXXWNvx9gD1JcMHeMPGZreuS3L4s357k7Svjd3T38939RJLDSa4e+PsAALvG6X5mq5N8tapeTPJf\nuvtgkvO7+6nl/h8nOX9Z3pfkvpV1n1zGfklV3ZDkhtOcG6foqvMuXfcUAGDPOd3YenN3H6mqf5Tk\nnqr669U7u7urqk/1SZdoO5gkW1kfAGC3OK3DiN19ZPn5TJIvZ+Ow4NNVdUGSLD+fWR5+JMlFK6tf\nuIwBvKwd78xDZyTC3rDlPVtV9atJXtHdP12Wfy/Jv01yd5Lrk9y6/LxrWeXuJH9RVR9P8mtJLkly\n/2nMHWDPEFawd53OYcTzk3y5ql56nr/o7v9WVQ8kubOq3pPkh0nemSTd/WhV3ZnksSQvJLmxu188\nrdkDAOxyW46t7v5+kt84xvj/TvI7x1nnI0k+stW/CQBwpnG5HgCAQWILAGCQ2AIAGCS2AAAGiS0A\ngEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0A\ngEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0A\ngEFiCwBg0NnrngCwNbfcdGDdUwBgE+zZAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFi\nCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGHT2uifA7vHAs4fW\nPQXgDHXVeZeuewqwa9mzBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBI\nbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBI\nbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAoJPGVlV9\npqqeqapHVsZeV1X3VNV3l5/nrtx3c1UdrqpDVfXWlfErqurh5b5PVFVt/8sBANhdNrNn67NJrj1q\n7ECSe7v7kiT3Lr+nqi5Lsj/JG5Z1PllVZy3r3JbkvUkuWW5HPycAwJ5z0tjq7q8n+clRw9cluX1Z\nvj3J21fG7+ju57v7iSSHk1xdVRckeW1339fdneRzK+sAAOxZZ29xvfO7+6ll+cdJzl+W9yW5b+Vx\nTy5jP1uWjx4/pqq6IckNW5wbAMCusdXY+oXu7qrq7ZjMynMeTHIwSbb7uQHgTHXLTQfWPQW2YKtn\nIz69HBrM8vOZZfxIkotWHnfhMnZkWT56HABgT9tqbN2d5Ppl+fokd62M76+qV1XVxdn4IPz9yyHH\n56rqmuUsxHevrAMAsGed9DBiVX0hyVuSnFdVTya5JcmtSe6sqvck+WGSdyZJdz9aVXcmeSzJC0lu\n7O4Xl6d6XzbObDwnyVeWGwDAnlYbJwfuXif6zNZun/uZ5oFnD617CsAZ6qrzLl33FOCUnegrP7t7\n274P1DfIAwAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAg\nsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAg\nsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwKDq7nXP4YSq6rgT3O1zBwB2r6o6\n7n3dffw7T5E9WwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCx\nBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPOXvcE4Ezx4Y/euu4pcIpuuenAuqcAYM8W\nAMAksQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QW\nAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QW\nAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDThpbVfWZqnqmqh5ZGftQVR2pqoeW\n29tW7ru5qg5X1aGqeuvK+BVV9fBy3yeqqrb/5QAA7C6b2bP12STXHmP8z7v78uX2V0lSVZcl2Z/k\nDcs6n6yqs5bH35bkvUkuWW7Hek4AgD3lpLHV3V9P8pNNPt91Se7o7ue7+4kkh5NcXVUXJHltd9/X\n3Z3kc0nevtVJAwCcKU7nM1vvr6rvLIcZz13G9iX50cpjnlzG9i3LR48fU1XdUFUPVtWDpzE/AIC1\n22ps3Zbk15NcnuSpJB/bthkl6e6D3X1ld1+5nc8LALDTthRb3f10d7/Y3T9P8qkkVy93HUly0cpD\nL1zGjizLR48DAOxpW4qt5TNYL3lHkpfOVLw7yf6qelVVXZyND8Lf391PJXmuqq5ZzkJ8d5K7TmPe\nAABnhLNP9oCq+kKStyQ5r6qeTHJLkrdU1eVJOskPkvxhknT3o1V1Z5LHkryQ5MbufnF5qvdl48zG\nc5J8ZbkBAOxpJ42t7n7XMYY/fYLHfyTJR44x/mCSN57S7AAAznC+QR4AYJDYAgAYJLYAAAad9DNb\nAHCmeeDZQ+uewsvGVedduu4p7Hr2bAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMqu5e9xxOqKqOO8HdPncA1uOBZw+tewovG1edd+m6p7BlVXXc+7r7+HeeInu2AAAGiS0A\ngEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAY5EtNAYCXJV9qCgCwB4gtAIBBYgsAYJDYAgAY\nJLYAAAaJLQCAQWILAGCQ2AIAGHT2uicAwPp8+KO3rnsKkCS55aYD657CGHu2AAAGiS0AgEFiCwBg\nkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEEnja2q\nuqiqvlZVj1XVo1X1gWX8dVV1T1V9d/l57so6N1fV4ao6VFVvXRm/oqoeXu77RFXVzMsCANgdNrNn\n64Ukf9TdlyW5JsmNVXVZkgNJ7u3uS5Lcu/ye5b79Sd6Q5Nokn6yqs5bnui3Je5Ncstyu3cbXAgCw\n65w0trr7qe7+1rL80ySPJ9mX5Lokty8Puz3J25fl65Lc0d3Pd/cTSQ4nubqqLkjy2u6+r7s7yedW\n1gEA2JPOPpUHV9Xrk7wpyTeSnN/dTy13/TjJ+cvyviT3raz25DL2s2X56PFj/Z0bktxwKnM70zzw\n7KF1T4E1uuq8S9c9BQB2yKY/IF9Vr07yxSQf7O7nVu9b9lT1dk2quw9295XdfeV2PScAwDpsKraq\n6pXZCK3Pd/eXluGnl0ODWX4+s4wfSXLRyuoXLmNHluWjxwEA9qzNnI1YST6d5PHu/vjKXXcnuX5Z\nvj7JXSvj+6vqVVV1cTY+CH//csjxuaq6ZnnOd6+sAwCwJ23mM1u/leQPkjxcVQ8tY3+S5NYkd1bV\ne5L8MMk7k6S7H62qO5M8lo0zGW/s7heX9d6X5LNJzknyleUGALBnnTS2uvt/JDne92H9znHW+UiS\njxxj/MEkbzyVCQIAnMl8gzwAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\nILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\nILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\nILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\nILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\nILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\nILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUAMEhsAQAMElsAAIPEFgDA\noJPGVlVdVFVfq6rHqurRqvrAMv6hqjpSVQ8tt7etrHNzVR2uqkNV9daV8Suq6uHlvk9UVc28LACA\n3eHsTTzmhSR/1N3fqqrXJPlmVd2z3Pfn3f0fVx9cVZcl2Z/kDUl+LclXq+qfdPeLSW5L8t4k30jy\nV0muTfKV7XkpAAC7z0n3bHX3U939rWX5p0keT7LvBKtcl+SO7n6+u59IcjjJ1VV1QZLXdvd93d1J\nPpfk7af9CgAAdrFT+sxWVb0+yZuysWcqSd5fVd+pqs9U1bnL2L4kP1pZ7cllbN+yfPT4sf7ODVX1\nYFU9eCrzAwDYbTZzGDFJUlWvTvLFJB/s7ueq6rYk/y5JLz8/luRfbsekuvtgkoPL3+3teE4Aftkt\nNx1Y9xRgz9vUnq2qemU2Quvz3f2lJOnup7v7xe7+eZJPJbl6efiRJBetrH7hMnZkWT56HABgz9rM\n2YiV5NNJHu/uj6+MX7DysHckeWRZvjvJ/qp6VVVdnOSSJPd391NJnquqa5bnfHeSu7bpdQAA7Eqb\nOYz4W0n+IMnDVfXQMvYnSd5VVZdn4zDiD5L8YZJ096NVdWeSx7JxJuONy5mISfK+JJ9Nck42zkJ0\nJiIAsKfVxomBu9eJPrO12+d+Ig88e2jdU2CNrjrv0nVPAeBl70Rf99nd2/ZdoL5BHgBgkNgCABgk\ntgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgk\ntgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgk\ntgAABoktAIBBYgsAYJDYAgAYJLYAAAZVd697DidUVced4G6fOwCwe1XVce/r7uPfeYrs2QIAGCS2\nAAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2\nAAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2\nAAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2\nAAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAadve4JnI6qWvcUAABOyJ4t\nAIBBYgsAYJDYAgAYdNLYqqq/V1X3V9W3q+rRqvrwMv66qrqnqr67/Dx3ZZ2bq+pwVR2qqreujF9R\nVQ8v932ifOgKANjjNrNn6/kk/6K7fyPJ5UmuraprkhxIcm93X5Lk3uX3VNVlSfYneUOSa5N8sqrO\nWp7rtiTvTXLJcrt2G18LAMCuc9LY6g3/d/n1lcutk1yX5PZl/PYkb1+Wr0tyR3c/391PJDmc5Oqq\nuiDJa7v7vu7uJJ9bWQcAYE/a1Fc/LHumvpnkHyf5z939jao6v7ufWh7y4yTnL8v7kty3svqTy9jP\nluWjx4/1925IcsPy6/NJHtnMPDkt5yV5dt2TeJmwrXeObb0zbOedY1vvjEu388k2FVvd/WKSy6vq\n7yf5clW98aj7u6p6uybV3QeTHEySqnqwu6/crufm2GznnWNb7xzbemfYzjvHtt4ZVfXgdj7fKZ2N\n2N3/J8nXsvFZq6eXQ4NZfj6zPOxIkotWVrtwGTuyLB89DgCwZ23mbMR/uOzRSlWdk+R3k/x1kruT\nXL887Pokdy3LdyfZX1WvqqqLs/FB+PuXQ47PVdU1y1mI715ZBwBgT9rMYcQLkty+fG7rFUnu7O6/\nrKr/meTOqnpPkh8meWeSdPejVXVnkseSvJDkxuUwZJK8L8lnk5yT5CvL7WQOnsLrYets551jW+8c\n23pn2M47x7beGdu6nWvjxEAAACb4BnkAgEFiCwBg0K6Nraq6drncz+GqOrDu+ewFVfWD5XJJD710\nWutWLrvE31VVn6mqZ6rqkZUxl7MacJxt/aGqOrK8rx+qqret3Gdbb0FVXVRVX6uqx5bLtH1gGfe+\n3mYn2Nbe19uo1n3pwe7edbckZyX5XpJfT/IrSb6d5LJ1z+tMvyX5QZLzjhr7syQHluUDSf7DsnzZ\nst1fleTi5b/HWet+DbvxluS3k/xmkkdOZ7smuT/JNUkqGyeP/P66X9tuux1nW38oyb8+xmNt661v\n5wuS/Oay/Jokf7NsT+/rndvW3tfbu50ryauX5Vcm+cayrXbkPb1b92xdneRwd3+/u/82yR3ZuAwQ\n2++ULru0hvntet399SQ/OWrY5awGHGdbH49tvUXd/VR3f2tZ/mmSx7NxxQ/v6212gm19PLb1FvSG\ntV16cLfG1r4kP1r5/biX9uGUdJKvVtU3a+OSSElyossu+W+wdae6Xfdlk5ez4pjeX1XfWQ4zvnQY\nwLbeBlX1+iRvysaeAO/rQUdt68T7eltV1VlV9VA2voT9nu7esff0bo0tZry5uy9P8vtJbqyq3169\nc6l03wWyzWzXcbdl4yMHlyd5KsnH1judvaOqXp3ki0k+2N3Prd7nfb29jrGtva+3WXe/uPw/8MJs\n7KX6pUsPZug9vVtj63iX/OE0dPeR5eczSb6cjcOCp3rZJTbH5ax2SHc/vfwj+vMkn8r/P9xtW5+G\nqnplNv7n//nu/tIy7H094Fjb2vt6Tq/h0oO7NbYeSHJJVV1cVb+SZH82LgPEFlXVr1bVa15aTvJ7\nSR7JKV52aWdnfUZzOasd8tI/lIt3ZON9ndjWW7Zsl08neby7P75yl/f1Njvetva+3l617ksPrvsM\ngROcOfC2bJyV8b0kf7ru+Zzpt2zsjv72cnv0pW2a5B8kuTfJd5N8NcnrVtb502X7H4qzWk60bb+Q\njd38P8vG8fv3bGW7JrkyG/+gfi/Jf8pyhQe3k27r/5rk4STfWf6BvMC2Pu3t/OZsHE75TpKHltvb\nvK93dFt7X2/vdv6nSf7Xsj0fSfJvlvEdeU+7XA8AwKDdehgRAGBPEFsAAIPEFgDAILEFADBIbAEA\nDBJbAACDxBYAwKD/ByyQmS8NSDOhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d3edf1aa20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def Draw_map4(Destination_x,Destination_y):\n",
    "    im4=Image.new('RGB',size=(3000,3000),color=(0,0,0))\n",
    "    draw=ImageDraw.Draw(im4,mode='RGB')\n",
    "    draw.rectangle((40,40,2960,2960),(255,255,255),(255,255,255))\n",
    "    draw.rectangle((500,800,1000,1200),(202,235,216),(202,235,216))\n",
    "    draw.rectangle((500,800,800,1200),(202,235,216),(202,235,216))\n",
    "    draw.rectangle((900,1300,1500,1800),(128,138,135),(128,138,135))\n",
    "    draw.rectangle((2000,800,800,1200),(202,235,216),(202,235,216))\n",
    "    draw.rectangle((500,250,800,500),(202,235,216),(202,235,216))\n",
    "    draw.rectangle((2500,200,2700,1000),(128,138,135),(128,138,135))\n",
    "    draw.rectangle((1100,300,2000,700),(128,138,135),(128,138,135))\n",
    "    draw.rectangle((2200,1550,2700,1850),(202,235,216),(202,235,216))\n",
    "    draw.rectangle((300,2000,1000,2700),(202,235,216),(202,235,216))\n",
    "    draw.rectangle((1900,1900,2800,2500),(128,138,135),(128,138,135))\n",
    "    draw.ellipse((Destination_x-20,Destination_y-20,Destination_x+20,Destination_y+20), (255,0,0),(255,0,0))\n",
    "    return im4\n",
    "im4=Draw_map4(1750,500)\n",
    "plt.imshow(im4)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Learning_rate=0.0001\n",
    "State_input=tf.placeholder(tf.float32,[None,Length_state])\n",
    "Action_input=tf.placeholder(tf.float32,[None,Length_action])\n",
    "Q_target_output=tf.placeholder(tf.float32,[None])\n",
    "\n",
    "Layer1_number=200\n",
    "w1=tf.Variable((tf.random_normal([Length_state,Layer1_number])))*0.01\n",
    "b1=tf.Variable(tf.zeros([1,Layer1_number]))\n",
    "l1=tf.nn.relu(tf.matmul(State_input,w1))+b1\n",
    "\n",
    "Layer12_number=20\n",
    "w12=tf.Variable((tf.random_normal([Layer1_number,Layer12_number])))*0.01\n",
    "b12=tf.Variable(tf.zeros([1,Layer12_number]))\n",
    "l12=tf.nn.relu(tf.matmul(l1,w12))+b12\n",
    "\n",
    "Layer2_number=Length_action\n",
    "w2=tf.Variable((tf.random_normal([Layer12_number,Layer2_number])))*0.01\n",
    "b2=tf.Variable(tf.zeros([1,Layer2_number]))\n",
    "l2=tf.matmul(l12,w2)+b2\n",
    "\n",
    "\n",
    "Q_value=tf.reduce_sum(tf.multiply(l2,Action_input),reduction_indices=1)\n",
    "Loss=tf.reduce_mean(tf.square(Q_value-Q_target_output))\n",
    "Optimizer=tf.train.AdamOptimizer(learning_rate).minimize(Loss)\n",
    "Predict=tf.argmax(l2,1)\n",
    "\n",
    "session=tf.InteractiveSession()\n",
    "session.run(tf.global_variables_initializer())\n",
    "\n",
    "def Save_memory(Current_state,Next_action,Next_state,Reward,Arrive):\n",
    "    Current_state_range=State_list[Current_state:Current_state+1]\n",
    "    Next_action_range=Action_list[Next_action:Next_action+1]\n",
    "    Next_state_range=State_list[Next_state:Next_state+1]\n",
    "    Store_memory.append((Current_state_range,Next_action_range,Next_state_range,Reward,Arrive))\n",
    "    if len(Store_memory)>Memory_size:\n",
    "        Store_memory.popleft()\n",
    "        \n",
    "def Experience_replay():\n",
    "    Minibatch=random.sample(Store_memory,Batch_size)\n",
    "    Batch_current_state = None\n",
    "    Batch_next_action = None\n",
    "    Batch_reward = None\n",
    "    Batch_next_state = None\n",
    "    Batch_arrive = None\n",
    "    \n",
    "    for index in range(Batch_size):\n",
    "        if Batch_current_state is None:\n",
    "            Batch_current_state=Minibatch[index][0]\n",
    "        elif Batch_current_state is not None:\n",
    "            Batch_current_state=np.vstack((Batch_current_state,Minibatch[index][0]))\n",
    "        \n",
    "        if Batch_next_action is None:\n",
    "            Batch_next_action=Minibatch[index][1]\n",
    "        elif Batch_next_action is not None:\n",
    "            Batch_next_action=np.vstack((Batch_next_action,Minibatch[index][1]))\n",
    "    \n",
    "        \n",
    "        if Batch_next_state is None:\n",
    "            Batch_next_state=Minibatch[index][2]\n",
    "        elif Batch_next_state is not None:\n",
    "            Batch_next_state=np.vstack((Batch_next_state,Minibatch[index][2]))\n",
    "        \n",
    "        if Batch_reward is None:\n",
    "            Batch_reward=Minibatch[index][3]\n",
    "        elif Batch_reward is not None:\n",
    "            Batch_reward=np.vstack((Batch_reward,Minibatch[index][3]))\n",
    "    \n",
    "        if Batch_arrive is None:\n",
    "            Batch_arrive=Minibatch[index][4]\n",
    "        elif Batch_arrive is not None:\n",
    "            Batch_arrive=np.vstack((Batch_arrive,Minibatch[index][4]))\n",
    "            \n",
    "    Q_next=session.run(l2,feed_dict={State_input:Batch_next_state})\n",
    "    Q_target_list=[]\n",
    "    for i in range(Batch_size):\n",
    "        Each_reward=Batch_reward[i][0]\n",
    "        Each_qvalue=Each_reward+Beta*np.max(Q_next[i])\n",
    "        if Each_reward<0:\n",
    "            Q_target_list.append(Each_reward)\n",
    "        else:\n",
    "            Q_target_list.append(Each_qvalue)\n",
    "\n",
    "    _,Cost,Reward=session.run([Q_value,Loss,Optimizer],feed_dict={State_input:Batch_current_state,\n",
    "                                                                         Action_input:Batch_next_action,\n",
    "                                                                         Q_target_output:Q_target_list})\n",
    "    if Action_times%5000==0:\n",
    "        print('Action_times=%d  Loss=%f'%(Action_times,Cost))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Set_destination(x_range,y_range):\n",
    "    x=np.random.randint(300,x_range-300)\n",
    "    y=np.random.randint(300,y_range-300)\n",
    "    return int(x),int(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Robot_destination(Destination_x,Destination_y,Robot_x,Robot_y,Robot_a):\n",
    "    Delta_x=Destination_x-Robot_x\n",
    "    Delta_y=Destination_y-Robot_y\n",
    "    Distance=np.sqrt(Delta_x*Delta_x+Delta_y*Delta_y)\n",
    "    Direct_angle=(np.arctan(Delta_y/Delta_x))*180/np.pi\n",
    "    if Delta_x<0:\n",
    "        Direct_angle=Direct_angle+180\n",
    "    True_angle=(Robot_a-Direct_angle)%360\n",
    "    if True_angle>180:\n",
    "        return 360-True_angle,'right',Distance\n",
    "    else:\n",
    "        return True_angle,'left',Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Is_Crash(Current_x,Current_y,im):\n",
    "    Crash=False\n",
    "    Degree=[-150,-120,-90,-60,-30,0,30,60,90,120,150,180]\n",
    "    Dis=np.arange(0,50,5)\n",
    "    for i in Dis:\n",
    "        for j in Degree:\n",
    "            x=Current_x+i*np.cos(j/180*np.pi)\n",
    "            y=Current_y+i*np.sin(j/180*np.pi)\n",
    "            if (im.getpixel((x,y)))==(0,0,0):\n",
    "                Crash=True\n",
    "                break\n",
    "        if Crash==True:\n",
    "                break\n",
    "    return Crash\n",
    "# Is_Crash(10,10,im4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Is_arrive(Current_x,Current_y,im):\n",
    "    Arrive=False\n",
    "    Crash=False\n",
    "    Degree=[-150,-120,-90,-60,-30,0,30,60,90,120,150,180]\n",
    "    Dis=np.arange(0,30,5)\n",
    "    for i in Dis:\n",
    "        for j in Degree:\n",
    "            x=Current_x+i*np.cos(j/180*np.pi)\n",
    "            y=Current_y+i*np.sin(j/180*np.pi)\n",
    "            if (im.getpixel((x,y)))==(0,0,0):\n",
    "                Crash=True\n",
    "                break\n",
    "            if (im.getpixel((x,y)))==(255,0,0):\n",
    "                Arrive=True\n",
    "                break\n",
    "        if Arrive==True or Crash==True:\n",
    "                break\n",
    "    return Arrive\n",
    "# print(Is_arrive(1750,500,im4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Random_start(x_range,y_range,im):\n",
    "    a=0\n",
    "    x=np.random.random()*x_range\n",
    "    y=np.random.random()*y_range\n",
    "    while(Is_Crash(x,y,im)==True or Is_arrive(x,y,im)==True):\n",
    "        x=np.random.random()*x_range\n",
    "        y=np.random.random()*y_range\n",
    "    return x,y,a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Height_level(Current_x,Current_y,Current_a,im):\n",
    "    Degree=[-60,-30,0,30,60]\n",
    "    Distance=np.arange(0,50,5)\n",
    "    Max_height=0\n",
    "    for i in Degree:\n",
    "        for j in Distance:\n",
    "            x=Current_x+i*np.cos((j+Current_a)/180*np.pi)\n",
    "            y=Current_y+i*np.sin((j+Current_a)/180*np.pi)\n",
    "            Height_level=0\n",
    "            if im.getpixel((x,y))==(255,255,255):\n",
    "                Height_level=0\n",
    "            elif im.getpixel((x,y))==(202,235,216):\n",
    "                Height_level=1\n",
    "            elif im.getpixel((x,y))==(128,138,135):\n",
    "                Height_level=2\n",
    "            Max_height=max(Height_level,Max_height)\n",
    "            if Max_height==2:\n",
    "                break\n",
    "        if Max_height==2:\n",
    "            break\n",
    "    return Height_level\n",
    "# Height_level(750,1000,im4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Output_state_index(True_angle,Left_right,Distance,Height):\n",
    "    if 0<=Distance<60:\n",
    "        Dis_level=0\n",
    "    elif 60<=Distance<180:\n",
    "        Dis_level=1\n",
    "    elif 180<=Distance<250:\n",
    "        Dis_level=2\n",
    "    else:\n",
    "        Dis_level=3\n",
    "    Angle_level=True_angle//30\n",
    "    if Left_right=='right':\n",
    "        Le_ri=1\n",
    "        State_number=int(Height*48+Dis_level+24*Le_ri+(5-Angle_level)*4)\n",
    "    elif Left_right=='left':\n",
    "        Le_ri=0\n",
    "        Angle_level=True_angle//30\n",
    "        State_number=int(Height*48+Dis_level+24*Le_ri+Angle_level*4)\n",
    "    return State_number,Dis_level,Angle_level\n",
    "# A,B,C=Output_state_index(80,'right',110,1)\n",
    "# States[A]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Choose_action(Current_state,Action_times):\n",
    "    Current_state_range=State_list[Current_state:Current_state+1]\n",
    "    Epsilon=Epsilon_final+(Epsilon_start-Epsilon_final)*np.exp(-1*Decay_Rate*Action_times)\n",
    "    if(np.random.random()<Epsilon):\n",
    "        Next_action=np.random.randint(Length_action)\n",
    "    else:\n",
    "        Action_qvalue_output=session.run(l2,feed_dict={State_input:Current_state_range})\n",
    "        Next_action=np.argmax(Action_qvalue_output)\n",
    "        \n",
    "    if Action_times%5000==0:\n",
    "        print('Action_times=%d  Epsilon=%f Epoche=%d'%(Action_times,Epsilon,Epoche))\n",
    "    return Next_action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Next_state_F(Robot_x,Robot_y,Robot_a,Destination_x,Destination_y,Action,im):\n",
    "    Arrive=False\n",
    "    Crash=False\n",
    "    Current_height=Height_level(Robot_x,Robot_y,Robot_a,im)\n",
    "    \n",
    "    if Action==0:\n",
    "        Robot_next_x=Robot_x+Velocity_tripod*np.cos(Robot_a/180*np.pi)\n",
    "        Robot_next_y=Robot_y+Velocity_tripod*np.sin(Robot_a/180*np.pi)\n",
    "        if Is_Crash(Robot_next_x,Robot_next_y,im)==True:\n",
    "            Crash=True\n",
    "        if Is_arrive(Robot_next_x,Robot_next_y,im)==True:\n",
    "            Arrive=True\n",
    "    elif Action==1:\n",
    "        Robot_next_x=Robot_x+Velocity_quadruped*np.cos(Robot_a/180*np.pi)\n",
    "        Robot_next_y=Robot_y+Velocity_quadruped*np.sin(Robot_a/180*np.pi)\n",
    "        if Is_Crash(Robot_next_x,Robot_next_y,im)==True:\n",
    "            Crash=True\n",
    "        if Is_arrive(Robot_next_x,Robot_next_y,im)==True:\n",
    "            Arrive=True\n",
    "    elif Action==2:\n",
    "        Robot_next_x=Robot_x+Velocity_onebyone*np.cos(Robot_a/180*np.pi)\n",
    "        Robot_next_y=Robot_y+Velocity_onebyone*np.sin(Robot_a/180*np.pi)\n",
    "        if Is_Crash(Robot_next_x,Robot_next_y,im)==True:\n",
    "            Crash=True\n",
    "        if Is_arrive(Robot_next_x,Robot_next_y,im)==True:\n",
    "            Arrive=True\n",
    "    elif Action==3:\n",
    "        Robot_a=Robot_a-45\n",
    "        Robot_next_x=Robot_x\n",
    "        Robot_next_y=Robot_y\n",
    "    elif Action==4:\n",
    "        Robot_a=Robot_a+45\n",
    "        Robot_next_x=Robot_x\n",
    "        Robot_next_y=Robot_y\n",
    "        \n",
    "   \n",
    "    \n",
    "    Next_true_angle,Next_le_ri,Next_distance=Robot_destination(Destination_x,Destination_y,Robot_next_x,Robot_next_y,Robot_a)\n",
    "    Next_height=Height_level(Robot_next_x,Robot_next_y,Robot_a,im)\n",
    "    Next_state_number,Next_distance_level,Next_angle_level=Output_state_index(Next_true_angle,Next_le_ri,Next_distance,Next_height)\n",
    "    \n",
    "    if (Current_height==2 and Action==0) or (Current_height==2 and Action==1):\n",
    "        Reward=-2\n",
    "    elif(Current_height==1 and Action==0) or (Current_height==1 and Action==2):\n",
    "        Reward=-2\n",
    "    elif(Current_height==0 and Action==1) or (Current_height==0 and Action==2):\n",
    "        Reward=-2\n",
    "    else:\n",
    "        if Arrive==False:\n",
    "            Reward=(-Next_distance_level-Next_angle_level*0.5)*0.01\n",
    "        else:\n",
    "            Reward=5\n",
    "    return Robot_next_x,Robot_next_y,Robot_a,Reward,Next_state_number,Arrive,Crash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Run_function():\n",
    "    global Epoche\n",
    "    global Action_times\n",
    "    global Vector_x\n",
    "    global Vector_y\n",
    "    Crash_=False\n",
    "    Arrive_=False\n",
    "    Destination_x,Destination_y=Set_destination(3000,3000)\n",
    "    im4=Draw_map4(Destination_x,Destination_y)\n",
    "#     plt.imshow(im4)\n",
    "#     plt.show()\n",
    "    Current_x,Current_y,Current_a=Random_start(3000,3000,im4)\n",
    "    Epoche_false=0\n",
    "    while Epoche_false<200:\n",
    "        True_a,Le_ri,Distance=Robot_destination(Destination_x,Destination_y,Current_x,Current_y,Current_a)\n",
    "        Height_level_=Height_level(Current_x,Current_y,Current_a,im4)\n",
    "        Current_state_number,Distance_level,Angle_level=Output_state_index(True_a,Le_ri,Distance,Height_level_)\n",
    "        Next_action=Choose_action(Current_state_number,Action_times)\n",
    "        Next_x,Next_y,Next_a,Reward,Next_state_number,Arrive,Crash=Next_state_F(Current_x,Current_y,Current_a,Destination_x,Destination_y,Next_action,im4)\n",
    "        if Crash==True:\n",
    "            Next_x,Next_y,Next_a=Random_start(3000,3000,im4)\n",
    "        else:\n",
    "            Save_memory(Current_state_number,Next_action,Next_state_number,Reward,Arrive)\n",
    "            if Action_times>Observe:\n",
    "                Experience_replay()\n",
    "            if Arrive:\n",
    "                Destination_x,Destination_y=Set_destination(3000,3000)\n",
    "                im4=Draw_map4(Destination_x,Destination_y)\n",
    "                Next_x,Next_y,Next_a=Random_start(3000,3000,im4)\n",
    "                Epoche_false+=1\n",
    "                Epoche+=1\n",
    "        Current_x=Next_x\n",
    "        Current_y=Next_y\n",
    "        Current_a=Next_a\n",
    "        Action_times+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action_times=425000  Epsilon=0.657232 Epoche=200\n",
      "Action_times=425000  Loss=0.000114\n",
      "Action_times=430000  Epsilon=0.654004 Epoche=204\n",
      "Action_times=430000  Loss=0.000048\n",
      "Action_times=435000  Epsilon=0.650792 Epoche=208\n",
      "Action_times=435000  Loss=0.004349\n",
      "Action_times=440000  Epsilon=0.647596 Epoche=209\n",
      "Action_times=440000  Loss=0.000067\n",
      "Action_times=445000  Epsilon=0.644416 Epoche=212\n",
      "Action_times=445000  Loss=0.037830\n",
      "Action_times=450000  Epsilon=0.641252 Epoche=215\n",
      "Action_times=450000  Loss=0.000228\n",
      "Action_times=455000  Epsilon=0.638103 Epoche=217\n",
      "Action_times=455000  Loss=0.005698\n",
      "Action_times=460000  Epsilon=0.634971 Epoche=221\n",
      "Action_times=460000  Loss=0.005393\n",
      "Action_times=465000  Epsilon=0.631854 Epoche=222\n",
      "Action_times=465000  Loss=0.000510\n",
      "Action_times=470000  Epsilon=0.628752 Epoche=223\n",
      "Action_times=470000  Loss=0.002386\n",
      "Action_times=475000  Epsilon=0.625666 Epoche=227\n",
      "Action_times=475000  Loss=0.000156\n",
      "Action_times=480000  Epsilon=0.622596 Epoche=229\n",
      "Action_times=480000  Loss=0.005954\n",
      "Action_times=485000  Epsilon=0.619540 Epoche=233\n",
      "Action_times=485000  Loss=0.004723\n",
      "Action_times=490000  Epsilon=0.616500 Epoche=235\n",
      "Action_times=490000  Loss=0.000120\n",
      "Action_times=495000  Epsilon=0.613475 Epoche=240\n",
      "Action_times=495000  Loss=0.246564\n",
      "Action_times=500000  Epsilon=0.610465 Epoche=242\n",
      "Action_times=500000  Loss=0.000351\n",
      "Action_times=505000  Epsilon=0.607471 Epoche=245\n",
      "Action_times=505000  Loss=0.001334\n",
      "Action_times=510000  Epsilon=0.604491 Epoche=248\n",
      "Action_times=510000  Loss=0.002176\n",
      "Action_times=515000  Epsilon=0.601526 Epoche=250\n",
      "Action_times=515000  Loss=0.003713\n",
      "Action_times=520000  Epsilon=0.598575 Epoche=253\n",
      "Action_times=520000  Loss=0.002770\n",
      "Action_times=525000  Epsilon=0.595640 Epoche=255\n",
      "Action_times=525000  Loss=0.001824\n",
      "Action_times=530000  Epsilon=0.592719 Epoche=257\n",
      "Action_times=530000  Loss=0.014424\n",
      "Action_times=535000  Epsilon=0.589813 Epoche=259\n",
      "Action_times=535000  Loss=0.001481\n",
      "Action_times=540000  Epsilon=0.586921 Epoche=263\n",
      "Action_times=540000  Loss=0.020879\n",
      "Action_times=545000  Epsilon=0.584043 Epoche=265\n",
      "Action_times=545000  Loss=0.000063\n",
      "Action_times=550000  Epsilon=0.581180 Epoche=269\n",
      "Action_times=550000  Loss=0.000115\n",
      "Action_times=555000  Epsilon=0.578332 Epoche=270\n",
      "Action_times=555000  Loss=0.019041\n",
      "Action_times=560000  Epsilon=0.575497 Epoche=270\n",
      "Action_times=560000  Loss=0.000245\n",
      "Action_times=565000  Epsilon=0.572677 Epoche=272\n",
      "Action_times=565000  Loss=0.000397\n",
      "Action_times=570000  Epsilon=0.569870 Epoche=274\n",
      "Action_times=570000  Loss=0.002866\n",
      "Action_times=575000  Epsilon=0.567078 Epoche=277\n",
      "Action_times=575000  Loss=0.000347\n",
      "Action_times=580000  Epsilon=0.564299 Epoche=279\n",
      "Action_times=580000  Loss=0.000683\n",
      "Action_times=585000  Epsilon=0.561535 Epoche=284\n",
      "Action_times=585000  Loss=0.001861\n",
      "Action_times=590000  Epsilon=0.558784 Epoche=286\n",
      "Action_times=590000  Loss=0.000376\n",
      "Action_times=595000  Epsilon=0.556047 Epoche=290\n",
      "Action_times=595000  Loss=0.000858\n",
      "Action_times=600000  Epsilon=0.553324 Epoche=294\n",
      "Action_times=600000  Loss=0.000206\n",
      "Action_times=605000  Epsilon=0.550614 Epoche=295\n",
      "Action_times=605000  Loss=0.015159\n",
      "Action_times=610000  Epsilon=0.547917 Epoche=298\n",
      "Action_times=610000  Loss=0.000638\n",
      "Action_times=615000  Epsilon=0.545234 Epoche=301\n",
      "Action_times=615000  Loss=0.000113\n",
      "Action_times=620000  Epsilon=0.542565 Epoche=302\n",
      "Action_times=620000  Loss=0.004861\n",
      "Action_times=625000  Epsilon=0.539909 Epoche=304\n",
      "Action_times=625000  Loss=0.001558\n",
      "Action_times=630000  Epsilon=0.537266 Epoche=305\n",
      "Action_times=630000  Loss=0.002493\n",
      "Action_times=635000  Epsilon=0.534636 Epoche=307\n",
      "Action_times=635000  Loss=0.000387\n",
      "Action_times=640000  Epsilon=0.532019 Epoche=312\n",
      "Action_times=640000  Loss=0.000045\n",
      "Action_times=645000  Epsilon=0.529416 Epoche=315\n",
      "Action_times=645000  Loss=0.000239\n",
      "Action_times=650000  Epsilon=0.526825 Epoche=319\n",
      "Action_times=650000  Loss=0.004451\n",
      "Action_times=655000  Epsilon=0.524248 Epoche=323\n",
      "Action_times=655000  Loss=0.002488\n",
      "Action_times=660000  Epsilon=0.521683 Epoche=326\n",
      "Action_times=660000  Loss=0.000776\n",
      "Action_times=665000  Epsilon=0.519131 Epoche=330\n",
      "Action_times=665000  Loss=0.061162\n",
      "Action_times=670000  Epsilon=0.516591 Epoche=333\n",
      "Action_times=670000  Loss=0.000154\n",
      "Action_times=675000  Epsilon=0.514065 Epoche=336\n",
      "Action_times=675000  Loss=0.000616\n",
      "Action_times=680000  Epsilon=0.511551 Epoche=339\n",
      "Action_times=680000  Loss=0.000128\n",
      "Action_times=685000  Epsilon=0.509049 Epoche=342\n",
      "Action_times=685000  Loss=0.014177\n",
      "Action_times=690000  Epsilon=0.506560 Epoche=343\n",
      "Action_times=690000  Loss=0.000048\n",
      "Action_times=695000  Epsilon=0.504084 Epoche=345\n",
      "Action_times=695000  Loss=0.000037\n",
      "Action_times=700000  Epsilon=0.501619 Epoche=347\n",
      "Action_times=700000  Loss=0.000236\n",
      "Action_times=705000  Epsilon=0.499167 Epoche=351\n",
      "Action_times=705000  Loss=0.000245\n",
      "Action_times=710000  Epsilon=0.496728 Epoche=351\n",
      "Action_times=710000  Loss=0.000025\n",
      "Action_times=715000  Epsilon=0.494300 Epoche=354\n",
      "Action_times=715000  Loss=0.000359\n",
      "Action_times=720000  Epsilon=0.491885 Epoche=358\n",
      "Action_times=720000  Loss=0.001643\n",
      "Action_times=725000  Epsilon=0.489481 Epoche=360\n",
      "Action_times=725000  Loss=0.024909\n",
      "Action_times=730000  Epsilon=0.487090 Epoche=365\n",
      "Action_times=730000  Loss=0.039607\n",
      "Action_times=735000  Epsilon=0.484710 Epoche=367\n",
      "Action_times=735000  Loss=0.000138\n",
      "Action_times=740000  Epsilon=0.482343 Epoche=371\n",
      "Action_times=740000  Loss=0.003173\n",
      "Action_times=745000  Epsilon=0.479987 Epoche=376\n",
      "Action_times=745000  Loss=0.000089\n",
      "Action_times=750000  Epsilon=0.477643 Epoche=379\n",
      "Action_times=750000  Loss=0.000423\n",
      "Action_times=755000  Epsilon=0.475311 Epoche=381\n",
      "Action_times=755000  Loss=0.000067\n",
      "Action_times=760000  Epsilon=0.472990 Epoche=383\n",
      "Action_times=760000  Loss=0.088850\n",
      "Action_times=765000  Epsilon=0.470681 Epoche=384\n",
      "Action_times=765000  Loss=0.001024\n",
      "Action_times=770000  Epsilon=0.468383 Epoche=387\n",
      "Action_times=770000  Loss=0.000582\n",
      "Action_times=775000  Epsilon=0.466097 Epoche=389\n",
      "Action_times=775000  Loss=0.000124\n",
      "Action_times=780000  Epsilon=0.463822 Epoche=391\n",
      "Action_times=780000  Loss=0.003290\n",
      "Action_times=785000  Epsilon=0.461559 Epoche=394\n",
      "Action_times=785000  Loss=0.000208\n",
      "Action_times=790000  Epsilon=0.459306 Epoche=399\n",
      "Action_times=790000  Loss=0.001175\n"
     ]
    }
   ],
   "source": [
    "Run_function()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L60D0/30/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L180D0/30/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L250D0/30/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L300D0/30/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L60D30/60/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>L180D30/60/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>L250D30/60/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>L300D30/60/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>L60D60/90/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>L180D60/90/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>L250D60/90/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>L300D60/90/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>L60D90/120/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>L180D90/120/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>L250D90/120/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>L300D90/120/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>L60D120/150/H0</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>L180D120/150/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>L250D120/150/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>L300D120/150/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>L60D150/180/H0</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>L180D150/180/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>L250D150/180/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>L300D150/180/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>L60D180/210/H0</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>L180D180/210/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>L250D180/210/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>L300D180/210/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>L60D210/240/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>L180D210/240/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>L250D210/240/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>L300D210/240/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>L60D240/270/H0</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>L180D240/270/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>L250D240/270/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>L300D240/270/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>L60D270/300/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>L180D270/300/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>L250D270/300/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>L300D270/300/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>L60D300/330/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>L180D300/330/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>L250D300/330/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>L300D300/330/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>L60D330/360/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>L180D330/360/H0</td>\n",
       "      <td>FrontH0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>L250D330/360/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>L300D330/360/H0</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>L60D0/30/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>L180D0/30/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>L250D0/30/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>L300D0/30/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>L60D30/60/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>L180D30/60/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>L250D30/60/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>L300D30/60/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>L60D60/90/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>L180D60/90/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>L250D60/90/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>L300D60/90/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>L60D90/120/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>L180D90/120/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>L250D90/120/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>L300D90/120/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>L60D120/150/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>L180D120/150/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>L250D120/150/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>L300D120/150/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>L60D150/180/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>L180D150/180/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>L250D150/180/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>L300D150/180/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>L60D180/210/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>L180D180/210/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>L250D180/210/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>L300D180/210/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>L60D210/240/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>L180D210/240/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>L250D210/240/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>L300D210/240/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>L60D240/270/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>L180D240/270/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>L250D240/270/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>L300D240/270/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>L60D270/300/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>L180D270/300/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>L250D270/300/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>L300D270/300/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>L60D300/330/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>L180D300/330/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>L250D300/330/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>L300D300/330/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>L60D330/360/H1</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>L180D330/360/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>L250D330/360/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>L300D330/360/H1</td>\n",
       "      <td>FrontH1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>L60D0/30/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>L180D0/30/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>L250D0/30/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>L300D0/30/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>L60D30/60/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>L180D30/60/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>L250D30/60/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>L300D30/60/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>L60D60/90/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>L180D60/90/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>L250D60/90/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>L300D60/90/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>L60D90/120/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>L180D90/120/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>L250D90/120/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>L300D90/120/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>L60D120/150/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>L180D120/150/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>L250D120/150/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>L300D120/150/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>L60D150/180/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>L180D150/180/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>L250D150/180/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>L300D150/180/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>L60D180/210/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>L180D180/210/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>L250D180/210/H2</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>L300D180/210/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>L60D210/240/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>L180D210/240/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>L250D210/240/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>L300D210/240/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>L60D240/270/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>L180D240/270/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>L250D240/270/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>L300D240/270/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>L60D270/300/H2</td>\n",
       "      <td>left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>L180D270/300/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>L250D270/300/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>L300D270/300/H2</td>\n",
       "      <td>right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>L60D300/330/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>L180D300/330/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>L250D300/330/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>L300D300/330/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>L60D330/360/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>L180D330/360/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>L250D330/360/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>L300D330/360/H2</td>\n",
       "      <td>FrontH2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   0        1\n",
       "0        L60D0/30/H0  FrontH0\n",
       "1       L180D0/30/H0    right\n",
       "2       L250D0/30/H0    right\n",
       "3       L300D0/30/H0    right\n",
       "4       L60D30/60/H0  FrontH0\n",
       "5      L180D30/60/H0    right\n",
       "6      L250D30/60/H0    right\n",
       "7      L300D30/60/H0    right\n",
       "8       L60D60/90/H0  FrontH0\n",
       "9      L180D60/90/H0    right\n",
       "10     L250D60/90/H0    right\n",
       "11     L300D60/90/H0    right\n",
       "12     L60D90/120/H0  FrontH0\n",
       "13    L180D90/120/H0    right\n",
       "14    L250D90/120/H0    right\n",
       "15    L300D90/120/H0    right\n",
       "16    L60D120/150/H0     left\n",
       "17   L180D120/150/H0    right\n",
       "18   L250D120/150/H0    right\n",
       "19   L300D120/150/H0    right\n",
       "20    L60D150/180/H0     left\n",
       "21   L180D150/180/H0    right\n",
       "22   L250D150/180/H0    right\n",
       "23   L300D150/180/H0    right\n",
       "24    L60D180/210/H0     left\n",
       "25   L180D180/210/H0    right\n",
       "26   L250D180/210/H0    right\n",
       "27   L300D180/210/H0    right\n",
       "28    L60D210/240/H0  FrontH0\n",
       "29   L180D210/240/H0    right\n",
       "30   L250D210/240/H0    right\n",
       "31   L300D210/240/H0    right\n",
       "32    L60D240/270/H0     left\n",
       "33   L180D240/270/H0    right\n",
       "34   L250D240/270/H0    right\n",
       "35   L300D240/270/H0    right\n",
       "36    L60D270/300/H0  FrontH0\n",
       "37   L180D270/300/H0    right\n",
       "38   L250D270/300/H0    right\n",
       "39   L300D270/300/H0    right\n",
       "40    L60D300/330/H0  FrontH0\n",
       "41   L180D300/330/H0    right\n",
       "42   L250D300/330/H0    right\n",
       "43   L300D300/330/H0    right\n",
       "44    L60D330/360/H0  FrontH0\n",
       "45   L180D330/360/H0  FrontH0\n",
       "46   L250D330/360/H0    right\n",
       "47   L300D330/360/H0    right\n",
       "48       L60D0/30/H1     left\n",
       "49      L180D0/30/H1  FrontH1\n",
       "50      L250D0/30/H1  FrontH1\n",
       "51      L300D0/30/H1  FrontH1\n",
       "52      L60D30/60/H1     left\n",
       "53     L180D30/60/H1  FrontH1\n",
       "54     L250D30/60/H1  FrontH1\n",
       "55     L300D30/60/H1  FrontH1\n",
       "56      L60D60/90/H1     left\n",
       "57     L180D60/90/H1  FrontH1\n",
       "58     L250D60/90/H1  FrontH1\n",
       "59     L300D60/90/H1  FrontH1\n",
       "60     L60D90/120/H1     left\n",
       "61    L180D90/120/H1  FrontH1\n",
       "62    L250D90/120/H1  FrontH1\n",
       "63    L300D90/120/H1  FrontH1\n",
       "64    L60D120/150/H1     left\n",
       "65   L180D120/150/H1  FrontH1\n",
       "66   L250D120/150/H1  FrontH1\n",
       "67   L300D120/150/H1  FrontH1\n",
       "68    L60D150/180/H1     left\n",
       "69   L180D150/180/H1  FrontH1\n",
       "70   L250D150/180/H1     left\n",
       "71   L300D150/180/H1  FrontH1\n",
       "72    L60D180/210/H1     left\n",
       "73   L180D180/210/H1  FrontH1\n",
       "74   L250D180/210/H1     left\n",
       "75   L300D180/210/H1  FrontH1\n",
       "76    L60D210/240/H1     left\n",
       "77   L180D210/240/H1  FrontH1\n",
       "78   L250D210/240/H1  FrontH1\n",
       "79   L300D210/240/H1  FrontH1\n",
       "80    L60D240/270/H1     left\n",
       "81   L180D240/270/H1  FrontH1\n",
       "82   L250D240/270/H1  FrontH1\n",
       "83   L300D240/270/H1  FrontH1\n",
       "84    L60D270/300/H1     left\n",
       "85   L180D270/300/H1  FrontH1\n",
       "86   L250D270/300/H1  FrontH1\n",
       "87   L300D270/300/H1  FrontH1\n",
       "88    L60D300/330/H1     left\n",
       "89   L180D300/330/H1  FrontH1\n",
       "90   L250D300/330/H1  FrontH1\n",
       "91   L300D300/330/H1  FrontH1\n",
       "92    L60D330/360/H1     left\n",
       "93   L180D330/360/H1  FrontH1\n",
       "94   L250D330/360/H1  FrontH1\n",
       "95   L300D330/360/H1  FrontH1\n",
       "96       L60D0/30/H2     left\n",
       "97      L180D0/30/H2  FrontH2\n",
       "98      L250D0/30/H2  FrontH2\n",
       "99      L300D0/30/H2  FrontH2\n",
       "100     L60D30/60/H2  FrontH2\n",
       "101    L180D30/60/H2  FrontH2\n",
       "102    L250D30/60/H2  FrontH2\n",
       "103    L300D30/60/H2  FrontH2\n",
       "104     L60D60/90/H2     left\n",
       "105    L180D60/90/H2  FrontH2\n",
       "106    L250D60/90/H2  FrontH2\n",
       "107    L300D60/90/H2  FrontH2\n",
       "108    L60D90/120/H2     left\n",
       "109   L180D90/120/H2  FrontH2\n",
       "110   L250D90/120/H2  FrontH2\n",
       "111   L300D90/120/H2  FrontH2\n",
       "112   L60D120/150/H2     left\n",
       "113  L180D120/150/H2  FrontH2\n",
       "114  L250D120/150/H2  FrontH2\n",
       "115  L300D120/150/H2  FrontH2\n",
       "116   L60D150/180/H2     left\n",
       "117  L180D150/180/H2  FrontH2\n",
       "118  L250D150/180/H2  FrontH2\n",
       "119  L300D150/180/H2  FrontH2\n",
       "120   L60D180/210/H2     left\n",
       "121  L180D180/210/H2  FrontH2\n",
       "122  L250D180/210/H2    right\n",
       "123  L300D180/210/H2  FrontH2\n",
       "124   L60D210/240/H2     left\n",
       "125  L180D210/240/H2  FrontH2\n",
       "126  L250D210/240/H2  FrontH2\n",
       "127  L300D210/240/H2  FrontH2\n",
       "128   L60D240/270/H2     left\n",
       "129  L180D240/270/H2  FrontH2\n",
       "130  L250D240/270/H2  FrontH2\n",
       "131  L300D240/270/H2  FrontH2\n",
       "132   L60D270/300/H2     left\n",
       "133  L180D270/300/H2  FrontH2\n",
       "134  L250D270/300/H2  FrontH2\n",
       "135  L300D270/300/H2    right\n",
       "136   L60D300/330/H2  FrontH2\n",
       "137  L180D300/330/H2  FrontH2\n",
       "138  L250D300/330/H2  FrontH2\n",
       "139  L300D300/330/H2  FrontH2\n",
       "140   L60D330/360/H2  FrontH2\n",
       "141  L180D330/360/H2  FrontH2\n",
       "142  L250D330/360/H2  FrontH2\n",
       "143  L300D330/360/H2  FrontH2"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "After_Q=session.run(l2,feed_dict={State_input:State_list})\n",
    "Max_qaction=np.argmax(After_Q,axis=1)\n",
    "Show_table=pd.DataFrame([States,Max_qaction]).T\n",
    "Show_table[Show_table==0]='FrontH0'\n",
    "Show_table[Show_table==1]='FrontH1'\n",
    "Show_table[Show_table==2]='FrontH2'\n",
    "Show_table[Show_table==3]='left'\n",
    "Show_table[Show_table==4]='right'\n",
    "Show_table\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
